package ngram_converter;

import ngram_converter.Converter.FileExistsOption;
import ngram.writer.NGramFileWriter;
import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import main.errorUpdate.ErrorUpdate;
import main.textUpdate.OutputUpdate;
import ngram.util.IOUtil;
import ngram.util.StringUtil;
import ngram.reader.TextFileReader;
import ngram.writer.V2FileWriter;
import stringFunctions.Stringify;
import swingExtended.progressPanel.ProgressPanel;

/**
 * Input: Google original n-gram
 * Output: Custom n-gram format version 2
 * 
 * @author Shimu
 * @date 9-Nov-2013
 */
public class Converter_V3 extends Converter {

    public Converter_V3(ProgressPanel panel, File inputDirectory, File outputDirectory,
            FileExistsOption option) {
        super(panel, inputDirectory, outputDirectory, option, "v2");
    }

    private void writeNGram(NGramFileWriter writer, String n_gram)
            throws IOException {
        // Don't write anything if the countInfo is empty
        if (this.YEARS.isEmpty()) {
            return;

        } else {
            writer.writeNGram(n_gram, YEARS, MATCH_COUNTS, VOLUME_COUNTS);
        }

        // Clear the information to use for the next n_gram
        this.YEARS.resetQuick();
        this.MATCH_COUNTS.resetQuick();
        this.VOLUME_COUNTS.resetQuick();
    }

    @Override
    protected void convert(File inputFile, File outputFile) throws Exception {
        try (BufferedReader in = IOUtil.getBufferedReader(inputFile.getAbsolutePath());
                DataOutputStream out = IOUtil.getDataOutputStream(outputFile.getAbsolutePath())) {
            final TextFileReader reader = new TextFileReader(in);
            final NGramFileWriter writer = new V2FileWriter(out);

            // The previous n-gram
            String prev = "";
            boolean skipToNextNGram = false;

            // The total number of distinct n-grams read so far
            int ngramCount = 1;
            String n_gram;
            while (!isCancelled() && (n_gram = reader.readNGram()) != null) {
                if (n_gram.isEmpty()) {
                    publish(new OutputUpdate(PANEL, "WARNING: line " + reader.getLineCount() + " is blank! [Omitted]"));
                    continue;
                }

                if (!StringUtil.equals(n_gram, prev)) {
                    // We reached a new n-gram
                    // Write all the year, lineCount information from the previous
                    // n-gram, stored in the array countInfo
                    writeNGram(writer, prev);
                    prev = n_gram;
                    ngramCount++;
                    skipToNextNGram = false;

                } else if (skipToNextNGram) {
                    continue;
                }
                //assert(gram == 0 || n_gram.split("\\s").length == gram);

                int year = reader.readYear();
                long matchCount = reader.readMatchCount();
                long volumeCount = reader.readVolumeCount();

                // The years in the YEARS array should be monotonically
                // increasing, if it's not, then we have some invalid
                // statistics, which we will warn the user and not write
                // the rest to output
                if (YEARS.size() > 0 && YEARS.get(YEARS.size() - 1) >= year) {
                    publish(new OutputUpdate(PANEL,
                            "WARNING: ngram: " + n_gram + " (Hex: " + Stringify.toHexString(n_gram.getBytes(IOUtil.ENCODING))
                            + ") at line: " + reader.getLineCount() + " has some invalid years! [Omitted]"));
                    // Ignore all the years afterwards
                    skipToNextNGram = true;
                    continue;

                } else if (YEARS.size() < CAPACITY) {
                    // Only add to the list if we did not reach capacity
                    YEARS.add(year);
                    MATCH_COUNTS.add(matchCount);
                    VOLUME_COUNTS.add(volumeCount);

                } else {
                    publish(new ErrorUpdate(PANEL, String.format(
                            "Word: %s occurred in more than %d years!",
                            n_gram, CAPACITY)));
                }

                // The rest of the String, which is either:
                // Version 1: page_count TAB volume_count 
                // Version 2: volume_count                
            }

            // Write the last n-gram
            writeNGram(writer, prev);
        }



    }
}
